from tfidfManager import TFIDFRetriever
import json
from functools import partial
file_dir='/home/lxy/wxbdata/rule_replace_supplement.jsonl'
tfidf_params={'token_pattern':r"(?u)\b\w+\b",'stop_words':['有','的','东北大学','是']}

with open(file_dir,'r',encoding='utf-8') as f:
    data=list(map(partial(json.dumps,ensure_ascii=False),f.readlines()))

import jieba
# data=list(map(partial(jieba.cut_for_search),data))
print(data)
for d in data:
    print(jieba.cut_for_search(d))
# documents = JSONLoader(file_dir,jq_schema='.[]',content_key='question').load()
exit()
retriever = TFIDFRetriever.from_texts(data,tfidf_params=tfidf_params)
result = retriever.get_relevant_documents("如何带外校人进入学校？")
print(result)